From: Keir Fraser Date: Tue, 16 Jun 2009 10:17:28 +0000 (+0100) Subject: tools/xenballoond: add tmem capability to directed/self-ballooning X-Git-Tag: archive/raspbian/4.8.0-1+rpi1~1^2~13778 X-Git-Url: https://dgit.raspbian.org/%22http:/www.example.com/cgi/%22https://%22%22/%22http:/www.example.com/cgi/%22https:/%22%22?a=commitdiff_plain;h=78c4e40239a539a8df9529eaa7409abe464ab8f3;p=xen.git tools/xenballoond: add tmem capability to directed/self-ballooning and monitor tool This patch adds tmem support to the largely unknown/unused xenballoond scripts that implement both self-ballooning and a foundation for directed-ballooning. Tmem and automated ballooning are highly complementary in that, when ballooning is over-aggressive, paging and swapping can increase noticably. Precache preserves evicted pages that may be needed again soon (thus eliminating disk reads) and preswap provides memory-based swapping that occurs if ballooning is insufficiently responsive to a sudden increase in activity and memory demand (thus eliminating disk writes and reads). There are two changes in this patch: 1) The xenballoond service is a convenient place to implement userland "preswap shrinking".** 2) The xenballoon-monitor script is a convenient place to report (and view with "watch -d") the frequent memory rebalancing that results from tmem usage on a busy system. Note that for best results (and for the monitor script to work), the xenstore-* tools should be installed on each guest. (** In a disk-based swap device, stale pages are often left on-disk even after they are no longer needed or valid; they are simply overwritten if/when the disk blocks are needed again, which may be a very long time. Preswap behaves much like a disk, but uses precious pages of memory that count against a guest's memory allocation; thus stale pages are very undesirable. Preswap shrinking periodically attempts to remove stale pages from preswap by using a sysfs interface created by the linux-side tmem patch.) Signed-off-by: Dan Magenheimer --- diff --git a/tools/xenballoon/xenballoon-monitor b/tools/xenballoon/xenballoon-monitor index 1aaaa1e9ec..c23cd39c83 100644 --- a/tools/xenballoon/xenballoon-monitor +++ b/tools/xenballoon/xenballoon-monitor @@ -2,6 +2,7 @@ # # xenballoon-monitor - monitor certain stats from xenballoond # (run in dom0 with "watch -d xenballoon-monitor" for xentop-like output) +# updated 090610 to include tmem stats # # Copyright (C) 2009 Oracle Corporation and/or its affiliates. # All rights reserved @@ -9,10 +10,12 @@ # # Hint: Use "xm sched-credit -d 0 -w 2000" to watch on heavily loaded machines # -echo "id mem-kb tgt-kb commit swapin swapout pgin pgout active(sec)" +TMEMTMP=$(/bin/mktemp -q /tmp/xenballoon-monitor.XXXXXX) +echo "id mem-kb tgt-kb commit swapin swapout pgin pgout preswap precache" for i in `xenstore-list /local/domain`; do if [ "$i" -ne 0 ]; then - tot=0; tgt=0; sin=0; sout=0; pgin=0; pgout=0; cmt=0; up=0; idle=0; act=0; + tot=0; tgt=0; sin=0; sout=0; pgin=0; pgout=0; cmt=0; up=0; idle=0; + act=0; preswap=0; precache=0 if xenstore-exists /local/domain/$i/memory/meminfo; then tot=`xenstore-read /local/domain/$i/memory/meminfo | grep MemTotal \ | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` @@ -23,21 +26,34 @@ for i in `xenstore-list /local/domain`; do tgt=`xenstore-read /local/domain/$i/memory/selftarget` fi if xenstore-exists /local/domain/$i/memory/vmstat; then - sin=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpin \ - | cut -d" " -f2` - sout=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpout \ - | cut -d" " -f2` - pgin=`xenstore-read /local/domain/$i/memory/vmstat | grep pgpgin \ - | cut -d" " -f2` - pgout=`xenstore-read /local/domain/$i/memory/vmstat | grep pgout \ - | cut -d" " -f2` + sin=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \ + | grep pswpin | cut -d" " -f2) + sout=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \ + | grep pswpout | cut -d" " -f2) + pgin=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \ + | grep pgpgin | cut -d" " -f2) + pgout=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \ + | grep pgout | cut -d" " -f2) fi - if xenstore-exists /local/domain/$i/memory/uptime; then - up=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f1` - idle=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f2` - act=`echo $up - $idle | bc -iq` + xm tmem-list --all --long > $TMEMTMP + precache=`grep "C=CI:$i" $TMEMTMP | sed 's/C=CI.*Ec:\([0-9][0-9]*\).*/\1/'` + if xenstore-exists /local/domain/$i/memory/preswap; then + preswap=`xenstore-read /local/domain/$i/memory/preswap` + printf "%2d %8d%8d%8d%7d%8d%9d%9d%9d%9d\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout $preswap $precache + else + printf "%2d %8d%8d%8d%9d%9d%10d%10d\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout fi - printf "%2d %8d%8d%8d%9d%9d%10d%10d%10.2f\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout $act fi done -echo Free memory: `xm info | grep free | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` MB +echo -n Free memory: `xm info | grep free | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` MiB " " +tmem_free_pages=`grep "G=" $TMEMTMP | sed 's/G=.*Ta:\([0-9][0-9]*\).*/\1/'` +if [ ! -z "$tmem_free_pages" ]; then + let "tmem_free_mb=$tmem_free_pages/256" + echo -n Idle tmem: $tmem_free_mb MiB " " +fi +tmem_eph_pages=`grep "G=" $TMEMTMP | sed 's/G=.*Ec:\([0-9][0-9]*\).*/\1/'` +if [ ! -z "$tmem_eph_pages" ]; then + let "tmem_eph_mb=$tmem_eph_pages/256" + echo -n Ephemeral tmem: $tmem_eph_mb MiB +fi +echo "" diff --git a/tools/xenballoon/xenballoon.conf b/tools/xenballoon/xenballoon.conf index 852abdc202..a1ca902aae 100644 --- a/tools/xenballoon/xenballoon.conf +++ b/tools/xenballoon/xenballoon.conf @@ -89,3 +89,48 @@ XENBALLOON_SEND_VMSTAT=1 # If xenbus is enabled, whether selfballooning or directed ballooning, # place the result of 'cat /proc/uptime" on xenbus at memory/uptime XENBALLOON_SEND_UPTIME=1 + +## Type: boolean +## Default: false +# +# If tmem is running, pages swapped to a swap disk may instead go +# into preswap. These pages may become stale (i.e. no longer need +# to be saved because, e.g., the process owning them has gone away) +# and stale pages use precious precious domain-allocated memory. +# Periodically try to reduce preswap to squeeze out stale pages +XENBALLOON_PRESWAP_SHRINK=false + +## Type: string +## Default: "/proc/sys/vm/preswap" +## (change to /sys/proc/vm/preswap in later kernels) +# +# If tmem and preswap are running, reading this file gives the number of +# pages currently in preswap. Writing it with N invokes the preswap_shrink +# routine to reduce preswap to N pages. +XENBALLOON_PRESWAP_SYSFILE=/proc/sys/vm/preswap + +## Type: integer (must be > 0) +## Default: 20 +# +# If tmem and preswap are running, attempts to reduce number of pages +# currently in preswap. For a value of n, 1/n of the pages will +# be attempted to be shrunk. +# If xenbus is enabled, may be overridden by {memory/preswaphysteresis} +XENBALLOON_PRESWAP_HYSTERESIS=20 + +## Type: integer (must be > 0) +## Default: 10 +# +# Number of ballooning intervals where preswap size remains at N pages +# before preswap shrinking is attempted. Also if shrinking fails to +# shrink to the target, counter resets to this value. +# If xenbus is enabled, may be overridden by {memory/preswapinertia} +XENBALLOON_PRESWAP_INERTIA=10 + +## Type: integer (0 or 1) +## Default: 1 +# +# If xenbus is enabled, whether selfballooning or directed ballooning, +# place the size of preswap (in pages) on xenbus at memory/preswap +XENBALLOON_SEND_PRESWAP=1 + diff --git a/tools/xenballoon/xenballoond b/tools/xenballoon/xenballoond index ec89c97df7..d8f43461de 100644 --- a/tools/xenballoon/xenballoond +++ b/tools/xenballoon/xenballoond @@ -5,9 +5,10 @@ # Written by: Dan Magenheimer # # xenballoond - In-guest engine for Xen memory ballooning -# Version: 080630 +# Original version: 080630 +# Updated 0906XX: add tmem preswap auto-shrinking # -# Two "policies" are implemented: +# Two self-ballooning "policies" are implemented: # - Selfballooning: Adjust memory periodically, with no (or little) input # from domain0. Target memory is determined solely by the # Committed_AS line in /proc/meminfo, but parameters may adjust @@ -149,6 +150,75 @@ send_memory_stats() { fi } + +curpreswappages() { + pages=$(cat $XENBALLOON_PRESWAP_SYSFILE) + RETVAL=$pages + return # value returned in RETVAL in pages +} + +preswaphysteresis() { + RETVAL=$XENBALLOON_PRESWAP_HYSTERESIS + if [ $xenstore_enabled = "true" ]; then + if xenstore-exists memory/preswaphysteresis ; then + RETVAL=`xenstore-read memory/preswaphysteresis` + fi + fi + return +} + +preswapinertia() { + RETVAL=$XENBALLOON_PRESWAP_INERTIA + if [ $xenstore_enabled = "true" ]; then + if xenstore-exists memory/preswapinertia ; then + RETVAL=`xenstore-read memory/preswapinertia` + fi + fi + return +} + +send_preswap_stats() { + if [ ! $xenstore_enabled = "true" ]; then + return + fi + curpreswappages + preswap_pgs=$RETVAL + if [ $XENBALLOON_SEND_PRESWAP ]; then + xenstore-write memory/preswap "$preswap_pgs" + fi +} + +shrink_preswap() { + if [ "$XENBALLOON_PRESWAP_SHRINK" = "false" ]; then + return + fi + if [ ! -f "$XENBALLOON_PRESWAP_SYSFILE" ]; then + return + fi + curpreswappages + preswaplast=$preswapnow + preswapnow=$RETVAL + if [ $preswapnow -eq 0 -o $preswapnow -ne $preswaplast ]; then + preswapinertia + preswapinertiacounter=$RETVAL + return + fi + let "preswapinertiacounter=$preswapinertiacounter-1" + if [ $preswapinertiacounter -ne 0 ]; then + return + fi + preswaphysteresis + preswaphys=$RETVAL + if [ $preswaphys -eq 0 ]; then + return + fi + let "tgtpreswappages=$(( $preswapnow - \ + ( $preswapnow / $preswaphys ) ))" + preswapinertia + preswapinertiacounter=$RETVAL + echo $tgtpreswappages > "$XENBALLOON_PRESWAP_SYSFILE" +} + if [ ! -f /proc/xen/balloon ]; then echo "$0: no balloon driver installed" exit 0 @@ -165,6 +235,7 @@ else echo "$0: missing /usr/bin/xenstore-* tools, disabling directed ballooning" xenstore_enabled=false fi +preswapnow=0 . /etc/sysconfig/xenballoon.conf @@ -194,7 +265,9 @@ do fi interval=$XENBALLOON_INTERVAL fi + shrink_preswap send_memory_stats + send_preswap_stats if [ $xenstore_enabled = "true" ]; then if xenstore-exists memory/interval ; then interval=`xenstore-read memory/interval` diff --git a/tools/xenballoon/xenballoond.init b/tools/xenballoon/xenballoond.init index bd2d928052..1cdf37f361 100644 --- a/tools/xenballoon/xenballoond.init +++ b/tools/xenballoon/xenballoond.init @@ -2,7 +2,7 @@ # # xenballoond Script to start and stop Xen ballooning daemon. # -# Copyright (C) 2008 Oracle Corporation and/or its affiliates. +# Copyright (C) 2009 Oracle Corporation and/or its affiliates. # All rights reserved. # Written by: Dan Magenheimer #